package dataset.generation;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Enumeration;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.List;

import opennlp.tools.sentdetect.SentenceDetectorME;
import opennlp.tools.sentdetect.SentenceModel;
import opennlp.tools.util.Span;

import org.itc.irst.tcc.sre.util.PorterStemmer;
import org.mitre.jawb.io.SgmlDocument;
import org.mitre.jawb.io.SgmlElement;

import edu.stanford.nlp.ling.HasWord;
import edu.stanford.nlp.tagger.maxent.MaxentTagger;

public class SentenceCreator {

	private static InputStream modelIn; 
	
	private static SentenceModel model;
	
	private static SentenceDetectorME sentenceDetector;

	private static Hashtable<String, char[]> abreviationsTable;

	private static Hashtable< String, String> invertedAbreviationsTable;

	private static Hashtable<String, char[]> abreviationsClassTable;

	private static Hashtable< String, String> invertedAbreviationsClassTable;

	private static Hashtable<String, String> TagToRelationTable;

	
	/**
	 * @param args
	 * @throws IOException 
	 * @throws FileNotFoundException 
	 * @throws ClassNotFoundException 
	 */
	public static void main(String[] args) throws FileNotFoundException, IOException, ClassNotFoundException {
		
		MaxentTagger tagger = new MaxentTagger(
                "/home/pjbarrio/Software/POSTagger/stanford-postagger-full-2011-06-19/models/left3words-wsj-0-18.tagger");
		
		modelIn = new FileInputStream("en-sent.bin");
		
		model = new SentenceModel(modelIn);

		sentenceDetector = new SentenceDetectorME(model);

		String[] Tags = {"NATURALDISASTER","LOCATION"};
		
		String[] abreviations = {"N","L"};
		
		String[] description = {"A","T"};
		
		abreviationsTable = new Hashtable<String, char[]>();
		abreviationsTable.put(Tags[0], abreviations[0].toCharArray());
		abreviationsTable.put(Tags[1], abreviations[1].toCharArray());
		
		abreviationsClassTable = new Hashtable<String, char[]>();
		abreviationsClassTable.put(Tags[0], description[0].toCharArray());
		abreviationsClassTable.put(Tags[1], description[1].toCharArray());
		
		invertedAbreviationsTable = new Hashtable<String, String>();
		invertedAbreviationsTable.put(abreviations[0],Tags[0]);
		invertedAbreviationsTable.put(abreviations[1],Tags[1]);
		
		invertedAbreviationsClassTable = new Hashtable<String, String>();
		invertedAbreviationsClassTable.put(description[0], description[0]);
		invertedAbreviationsClassTable.put(description[1], description[1]);
		
		String[] RelationTags = {"atLocation","locationOf"};
		
		TagToRelationTable = new Hashtable<String, String>();
		TagToRelationTable.put(Tags[0],RelationTags[0]);
		TagToRelationTable.put(Tags[1], RelationTags[1]);

		int[] splits = {0,1,2,3,4,5,6,7,8,9};
		
		for (int i = 0; i < splits.length; i++) {

			String directoryName = "/home/pjbarrio/Dataset/NaturalDisaster/RelatedData/reverbData/"+i+"/taggedtestingDirectory/";
			
			String outputFile = "/home/pjbarrio/Dataset/NaturalDisaster/RelatedData/reverbData/"+i+"/natural_disaster.xjsre.test";
	
			File directory = new File(directoryName);
			
			File[] files = directory.listFiles();
			
			System.setOut(new PrintStream(new File(outputFile)));
			
			System.setErr(new PrintStream(new File(outputFile + ".error")));
						
			int documentId = 0;
			
			for (File file : files) {
				
				if (!file.getName().endsWith(".txt"))
					continue;
				
				documentId++;
				
				System.err.println(file);
				
				SgmlDocument sgmlDocument = new SgmlDocument(new FileReader(file));
				
				Hashtable<String, List<SgmlElement>> sgmlElements = getSgmlElements(sgmlDocument,Tags);
				
				Hashtable<String, List<SgmlElement>> sgmlRelationElements = getSgmlElements(sgmlDocument, RelationTags);
				
				List<Segment> sentences = getRelevantSentences(sgmlDocument,Tags,sgmlElements); 
				
				int sentenceId = 0;
				
				//Stem
				for (Segment segment : sentences) {
					
					sentenceId++;
					
					List<SgmledSegment> combinations = generateCombinations(segment,Tags,sgmlElements);
					
					Hashtable<String, List<SgmlElement>> filteredRelations = filter(sgmlRelationElements, segment, RelationTags);
					
					int variationId = 0;
					
					for (SgmledSegment sgmledSegment : combinations) {
						
						variationId++;
						
						String hyphened = hyphenize(sgmledSegment);
						
						String reducedhyphened = hyphened.replaceAll("_+", "_");
						
						String[] POSTags = getPOSTags(tagger, reducedhyphened);
						
						String[] tokenWords = getWords(tagger, reducedhyphened);
						
						String[] stemmed = getStemmed(tokenWords);
	
						String[] entity_types = getEntitiesType(tagger,sgmledSegment,hyphened); 
						
						String[] entity_label = getEntitiesLabel(tagger,sgmledSegment,hyphened,filteredRelations,entity_types);
						
						if (containsTargetAndAgent(entity_label)){
							
							if (sameSize(POSTags.length,tokenWords.length,stemmed.length,entity_types.length,entity_label.length)){
								
								int classifiedAs = getClass(sgmledSegment,filteredRelations);
															
								System.out.println(classifiedAs + "\t" + generateId(documentId,sentenceId,variationId) + "\t" + generateBody(tokenWords,stemmed,POSTags,entity_types,entity_label));
								
							} else {
								
								System.err.println("ERROR-0: " + file.getName() + " - " + POSTags.length + " - " + tokenWords.length + " - " + stemmed.length + " - " + entity_types.length + " - " + entity_label.length);
								
								
							}
							
	
							
						} else {
							
							System.err.println("ERROR-1\t" + generateId(documentId,sentenceId,variationId) + "\t" + generateBody(tokenWords,stemmed,POSTags,entity_types,entity_label));
							
						}
						
					}
					
					
				}
				
				
				
				
			}

		}
			
	}

	private static boolean sameSize(int length1, int length2, int length3,
			int length4, int length5) {
		return (length1 == length2) && (length2 == length3) && (length3 == length4) && (length4 == length5);
	}

	private static boolean containsTargetAndAgent(String[] entity_label) {
		
		boolean containsAgent = false;
		boolean containsTarget = false;
		
		for (int i = 0; i < entity_label.length; i++) {
			
			if (entity_label[i].equals("T")){
				containsTarget = true;
				if (containsAgent) return true;
			}
			if (entity_label[i].equals("A")){
				containsAgent = true;
				if (containsTarget) return true;
			}
				
		}
		
		return false;
		
	}

	private static String generateBody(String[] tokenWords, String[] stemmed,
			String[] pOSTags, String[] entity_types, String[] entity_label) {
		
		String ret = "";
		
		for (int i = 0; i < tokenWords.length; i++) {
			
			if (entity_types[i] != "O"){
				stemmed[i] = tokenWords[i];
			}
			
			ret = ret + " " + i + "&&" + tokenWords[i] + "&&" + stemmed[i] + "&&" + pOSTags[i] + "&&" + entity_types[i] + "&&" + entity_label[i]; 
			
		}
		
		return ret.substring(1);
	}

	private static String generateId(int documentId, int sentenceId,
			int variationId) {
		return new String(documentId + "-" + sentenceId + "-" + variationId);
	}

	private static int getClass(SgmledSegment sgmledSegment,
			Hashtable<String, List<SgmlElement>> filteredRelations) {
		// returns 1 if the entities are related. 0 otherwise.
		
		SgmlElement fE = sgmledSegment.getRelevantSgmlElement(1);
		SgmlElement sE = sgmledSegment.getRelevantSgmlElement(2);
		
		List<SgmlElement> relevantElements = new ArrayList<SgmlElement>();
		
		relevantElements.add(fE);
		relevantElements.add(sE);
		
		List<SgmlElement> relevantRelation = new ArrayList<SgmlElement>();
		
		for (SgmlElement sgmlElement : relevantElements) {
			
			String lookingForTag = TagToRelationTable.get(sgmlElement.getGid());
			
			List<SgmlElement> relations = filteredRelations.get(lookingForTag);
			
			for (SgmlElement relationSgmlElement : relations) {
				
				if (matches(relationSgmlElement,sgmlElement)){
					relevantRelation.add(relationSgmlElement);
				}
				
			}
			
		}
		
		if (relevantRelation.size() == relevantElements.size()){
			
			if (sameRelation(relevantRelation)){
				return 1;
			}
			
		}
		
		return 0;
	}

	private static boolean sameRelation(List<SgmlElement> relevantRelation) {
		
		String attribute = relevantRelation.get(0).getAttribute("id");
		
		for (SgmlElement sgmlElement : relevantRelation) {
			
			if (!attribute.equals(sgmlElement.getAttribute("id"))){
				return false;
			}
			
		}
		
		return true;
	}

	private static boolean matches(SgmlElement relationSgmlElement,
			SgmlElement sgmlElement) {
		
		return (relationSgmlElement.getStart() >= sgmlElement.getStart() && relationSgmlElement.getEnd() <= sgmlElement.getEnd()) 
				|| (sgmlElement.getStart()>=relationSgmlElement.getStart() && sgmlElement.getEnd() <= relationSgmlElement.getEnd());
		
	}

	private static String[] getEntitiesLabel(MaxentTagger tagger,
			SgmledSegment sgmledSegment, String hyphened,
			Hashtable<String, List<SgmlElement>> filteredRelations, String[] entity_types) {
		
		SgmlElement fE = sgmledSegment.getRelevantSgmlElement(1);
		SgmlElement sE = sgmledSegment.getRelevantSgmlElement(2);
		
		List<SgmlElement> sorted = new ArrayList<SgmlElement>();
		
		sorted.add(fE);
		sorted.add(sE);
		
		char[] text = hyphened.toCharArray();
		
		for (SgmlElement sgmlElement : sorted) {
			
			char letter = getClassAbreviation(sgmlElement.getGid());
			
			for (int i = sgmlElement.getStart(); i < sgmlElement.getEnd(); i++) {
				
				int index = i-sgmledSegment.getOffset();
				
//				if (text[index] != '_')
				if (Character.isLetterOrDigit(text[index]))
					
					if (Character.isLowerCase(text[index]))
						text[index] = Character.toLowerCase(letter);
					else
						text[index] = letter;
				
			}
			
		}
		
		
		String reducedText = new String(text).replaceAll("_+", "_");
		
		String[] entities = getWords(tagger, new String(reducedText));
		
		String[] ret = new String[entities.length];
		
		for (int i = 0; i < entities.length; i++) {
			
			if (entity_types[i] == "O"){
				ret[i] = "O";
			}else{
			
				ret[i] = getEntityLabel(entities[i]);
			
			}
		}

		return ret;
		
	}

	private static String getEntityLabel(String word) {
		
		char[] letters = word.toCharArray();
		
		char letter = '\0';
		
		boolean same = true;
		
		for (int i = 0; i < letters.length-1; i++) {
			
//			if (letters[i] == '_' || letters[i+1] == '_')
			if (!Character.isLetterOrDigit(letters[i]) || !Character.isLetterOrDigit(letters[i+1]))
				continue;
			
			letter = letters[i];
			
			if (Character.toLowerCase(letters[i]) != Character.toLowerCase(letters[i+1])){
				same = false;
				break;
			}
				
		}
		
		if (same){
			
			String s = String.valueOf(Character.toUpperCase(letter));
			
			String ret = invertedAbreviationsClassTable.get(s);
			
			if (ret == null){
				return "O";
			}
			
			return ret;
		}
		
		return "O";
		
	}

	private static char getClassAbreviation(String gid) {
		return abreviationsClassTable.get(gid)[0];
	}

	private static String[] getEntitiesType(MaxentTagger tagger, SgmledSegment sgmledSegment, String hyphened) {
		
		List<SgmlElement> sorted = sgmledSegment.getSgmlElements();
		
		char[] text = hyphened.toCharArray();
		
		for (SgmlElement sgmlElement : sorted) {
			
			char letter = getAbreviation(sgmlElement.getGid());
			
			for (int i = sgmlElement.getStart(); i < sgmlElement.getEnd(); i++) {
				
				int index = i-sgmledSegment.getOffset();

//				if (text[index] != '_')
				if (Character.isLetterOrDigit(text[index])){
					
					if (Character.isLowerCase(text[index]))
						text[index] = Character.toLowerCase(letter);
					else
						text[index] = letter;
					
				}
			}
			
		}
		
		String reducedText = new String(text).replaceAll("_+", "_");
		
		String[] entities = getWords(tagger, new String(reducedText));
		
		String[] ret = new String[entities.length];
		
		for (int i = 0; i < entities.length; i++) {
			
			ret[i] = getEntityType(entities[i]);
			
		}

		return ret;
	}

	private static String getEntityType(String word) {
		
		char[] letters = word.toCharArray();
		
		boolean same = true;
		
		char letter = '\0';
		
		for (int i = 0; i < letters.length-1; i++) {

//			if (letters[i] == '_' || letters[i+1] == '_')
			if (!Character.isLetterOrDigit(letters[i]) || !Character.isLetterOrDigit(letters[i+1]))
				continue;
			
			letter = letters[i];
			
			if (Character.toLowerCase(letters[i]) != Character.toLowerCase(letters[i+1])){
				same = false;
				break;
			}
				
		}
		
		if (same){
			
			String s = String.valueOf(Character.toUpperCase(letter));
			
			String ret = invertedAbreviationsTable.get(s);
			
			if (ret == null){
				return "O";
			}
			
			return ret;
		}
		
		return "O";
	}

	private static char getAbreviation(String tag) {
		
		return abreviationsTable.get(tag)[0];
		
	}

	private static String[] getStemmed(String[] tokenWords) {
		
		String[] stemmed = new String[tokenWords.length];
		
		PorterStemmer porterStemmer = PorterStemmer.getStemmer();
		
		for (int i = 0; i < tokenWords.length; i++) {
			
			stemmed[i] = porterStemmer.stem(tokenWords[i]);
			
		}
		
		return stemmed;
	}

	private static String[] getPOSTags(MaxentTagger tagger, String hyphened) {
		
		String tagged = tagger.tagString(hyphened);
		
		String[] taggedElements = tagged.split(" ");
		
		String[] POSTags = new String[taggedElements.length];
		
		for (int i = 0; i < taggedElements.length; i++) {
			
			POSTags[i] = taggedElements[i].substring(taggedElements[i].lastIndexOf('/')+1);
			
		}
		
		return POSTags;
		
	}

	private static String[] getWords(MaxentTagger tagger, String text) {
		
		List<List<HasWord>> words = tagger.tokenizeText(new StringReader(text));
		
		List<String> ret = new ArrayList<String>();
		
		for (List<HasWord> list : words) {
			
			for (HasWord hasWord : list) {
								
				ret.add(hasWord.word());
				
			}
			
		}
		
		return ret.toArray(new String[0]);
	}

	private static List<SgmledSegment> generateCombinations(Segment segment, String[] tags, Hashtable<String, List<SgmlElement>> sgmlElements) {
		
		List<SgmledSegment> ret = new ArrayList<SgmledSegment>();
		
		Hashtable<String, List<SgmlElement>> containedSgmlElements = filter(sgmlElements,segment,tags);
	
		List<SgmlElement> sorted = sort(containedSgmlElements);
		
		List<SgmlElement> tag_0 = containedSgmlElements.get(tags[0]);
		
		List<SgmlElement> tag_1 = containedSgmlElements.get(tags[1]);
		
		for (SgmlElement sgmlElement_0 : tag_0) {
			
			for (SgmlElement sgmlElement_1 : tag_1) {
				
				SgmledSegment aux = new SgmledSegment(segment.getText(), segment.getOffset(), sorted);
				
				aux.addSgmlElement(sgmlElement_0,1);
				
				aux.addSgmlElement(sgmlElement_1,1);
				
				ret.add(aux);
				
			}
			
		}
		
		return ret;
		
	}

	private static List<SgmlElement> sort(
			Hashtable<String, List<SgmlElement>> containedSgmlElements) {
		
		List<SgmlElement> totalSgmlElements = new ArrayList<SgmlElement>();
		
		for(Enumeration<String> e = containedSgmlElements.keys();e.hasMoreElements();){
			totalSgmlElements.addAll(containedSgmlElements.get(e.nextElement()));
		}

		Collections.sort(totalSgmlElements, new Comparator<SgmlElement>() {

			@Override
			public int compare(SgmlElement arg0, SgmlElement arg1) {
				
				return Integer.valueOf(arg0.getStart()).compareTo(arg1.getStart());
				
			}
		});
		
		return totalSgmlElements;
	}

	private static Hashtable<String, List<SgmlElement>> filter(
			Hashtable<String, List<SgmlElement>> sgmlElements, Segment segment,
			String[] tags) {
		
		Hashtable<String, List<SgmlElement>> ret = new Hashtable<String, List<SgmlElement>>();
		
		for (int i = 0; i < tags.length; i++) {
			
			List<SgmlElement> elements = sgmlElements.get(tags[i]);
			
			ArrayList<SgmlElement> aux = new ArrayList<SgmlElement>();
			
			for (SgmlElement sgmlElement : elements) {
				
				if (contains(segment, sgmlElement)){
					aux.add(sgmlElement);
				}
				
			}
			
			ret.put(tags[i], aux);
			
		}
		
		return ret;
	}

	private static String hyphenize(SgmledSegment segment) {
		
//		System.err.println(segment.getText());
		
		char[] text = segment.getText().toCharArray();
		
		for (SgmlElement sgmlElement : segment.getSgmlElements()) {
			
			for (int i = sgmlElement.getStart(); i < sgmlElement.getEnd(); i++) {
				
				int index = i-segment.getOffset();
				
//				if (text[index] == ' ' || text[index] == '\r' || text[index] == '\n' || text[index] == '\t'){
				if (!Character.isLetterOrDigit(text[index])){
					text[index] = '_';
				}
				
			}
			
		}
		
		return new String(text);
	}

	private static List<Segment> getRelevantSentences(SgmlDocument sgmlDocument, String[] tags, Hashtable<String, List<SgmlElement>> sgmlElements) throws FileNotFoundException, IOException {
		
		String plainText = sgmlDocument.getSignalText();
		
		List<Segment> segments = getSegments(plainText);
		
		List<Segment> sentences = generateSentences(segments);
		
		List<Segment> fixedSentences = fixSentences(sentences,sgmlElements,tags);
		
		List<Segment> filteredSenteces = filterSentences(sgmlElements,fixedSentences,tags);
		
		return filteredSenteces;
	}

	private static List<Segment> fixSentences(List<Segment> segments,
			Hashtable<String, List<SgmlElement>> sgmlElements, String[] tags) {

		List<Segment> ret = new ArrayList<Segment>();

		for (int j = 0; j < segments.size(); j++) {
			
			Segment seg = segments.get(j);
			
			boolean changed = false;
			
			for (int i = 0; i < tags.length && !changed; i++) {
				
				List<SgmlElement> se = sgmlElements.get(tags[i]);
				
				for (SgmlElement sgmlElement : se) {
					
					if (contains(seg, sgmlElement) && !containsItAll(seg, sgmlElement)){
						
						String link = "";
						
						int initSec = segments.get(j+1).getOffset();
						int endPrev = seg.getOffset() + seg.getText().length();
						
						int add = initSec - endPrev;
						
						for (int k = 0; k < add; k++) {
						
							link = link + " ";
						
						}
						
						segments.set(j+1, new Segment(seg.getText() + link + segments.get(j+1).getText(), seg.getOffset()));
						
						changed = true;
						
						break;
						
					}
					
				}
				
			}
			
			if (!changed)
				ret.add(seg);
			
		}
		
		return ret;
		
	}

	private static List<Segment> filterSentences(
			Hashtable<String, List<SgmlElement>> sgmlElements,
			List<Segment> sentences, String[] tags) {
		// return sentences that have both tags.
		
		ArrayList<Segment> ret = new ArrayList<Segment>();
		
		for (Segment sentence : sentences) {
			
			if (contains(sentence,sgmlElements.get(tags[0])) && contains(sentence,sgmlElements.get(tags[1]))){
				ret.add(sentence);
			}
			
		}
		
		return ret;
	}

	private static boolean contains(Segment sentence, List<SgmlElement> list) {
		
		for (SgmlElement sgmlElement : list) {
			
			if (contains(sentence,sgmlElement)){
				return true;
			}
						
		}
		
		return false;
	}

	private static boolean containsItAll(Segment sentence, SgmlElement sgmlElement) {

		int start = sentence.getOffset();
		int end = start + sentence.getText().length();

		if (sgmlElement.getStart() >= start && sgmlElement.getEnd() <= end){
			
			return true;
			
		}
		
		return false;
		
	}
	
	private static boolean contains(Segment sentence, SgmlElement sgmlElement) {
		int start = sentence.getOffset();
		int end = start + sentence.getText().length();

		if (sgmlElement.getStart() >= start && sgmlElement.getStart() <= end){
			
			return true;
			
		}
		
		return false;
		
	}

	private static Hashtable<String, List<SgmlElement>> getSgmlElements(
			SgmlDocument sgmlDocument, String[] tags) {
		
		Hashtable<String, List<SgmlElement>> ret = new Hashtable<String, List<SgmlElement>>();
		
		for (int i = 0; i < tags.length; i++) {
			
			ArrayList<SgmlElement> aux = new ArrayList<SgmlElement>();
			
			for (Iterator iterator = sgmlDocument.iterator(); iterator.hasNext();) {
				
				SgmlElement sgmlElement = (SgmlElement) iterator.next();
				
				if (sgmlElement.getGid().equals(tags[i])){
					
					aux.add(sgmlElement);
					
				}
				
			}
			
			ret.put(tags[i],aux);
			
		}
		
		return ret;
		
	}

	private static List<Segment> generateSentences(List<Segment> contentParagraph) {
		
		ArrayList<Segment> ret = new ArrayList<Segment>();
		
		for (Segment segment : contentParagraph){
			
			ret.addAll(getSentences(segment));
			
		}

		return ret;
		
	}

	private static Collection<? extends Segment> getSentences(Segment segment) {
		
		ArrayList<Segment> ret = new ArrayList<Segment>();
		
		String sentences[] = sentenceDetector.sentDetect(segment.getText());
			
		Span[] spans = sentenceDetector.sentPosDetect(segment.getText());
		
		for (int i = 0; i < sentences.length; i++) {
			
			ret.add(new Segment(sentences[i],segment.getOffset() + spans[i].getStart()));
			
		}

		return ret;
		
	}

	
	private static List<Segment> getSegments(String text) throws IOException {
		
		BufferedReader br = new BufferedReader(new StringReader(text));
		
		ArrayList<Segment> ret = new ArrayList<Segment>();
		
	    String acum = "";
	    
	    String line = br.readLine();
	     
	    int offset = 0;
	    
	    int newOffset = 0;
	    
	    while (line != null){
	    	
	    	if (line.trim().equals("")){
	    		
	    		if (!acum.trim().equals("")){
	    				    			
	    			Segment s = new Segment(acum,offset);
	    			
	    			ret.add(s);
	    		
	    		}
	    		
	    		acum = "";
	    	
	    	} else {
	    		
	    		if (acum.trim().equals("")){  //First
	    			
	    			acum = line;
	    			
	    			offset = newOffset;
	    			
	    		} else {
		    		acum = acum + " " + line;
	    		}
	    		
	    	}
	    	
	    	newOffset += line.length() + 1;
	    	
	    	line = br.readLine();
	    
	    }
	    
	    if (!acum.trim().equals("")){
	    	
	    	Segment s = new Segment(acum,offset);
	    	
	    	ret.add(s);
	    	
	    }
	    
	    br.close();
	    
	    return ret;
		
	}

}
